using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

using System.Data;
using System.Threading;

namespace Com.Rf.Core
{
    public class Crawler
    {
        public Crawler(string url)
        {
            _url = url;
            AddToVisit(url);
        }

        private string _encoding = "GB2312";

        private string _url;

        public void SetEncoding(string encoding)
        {
            _encoding = encoding;
        }
        
        public void AddToVisit(string url)
        {
            DBHelper oDBHelper = new DBHelper();
            string sql;
            DataTable dt;

            url = url.Replace("'", "''").ToLower();
            sql = "select * from tb_Visit where Url='" + url + "'";
            dt = oDBHelper.Query(sql).Tables[0];
            if (dt.Rows.Count == 0)
            {
                sql = "insert into tb_Visit(Url,Visited) values('" + url + "','N')";
                oDBHelper.Execute(sql);
            }
        }

        virtual public void Visit(Page page)
        {

            AddToVisit(page.Url);
        }

        private void Start()
        {
            try
            {
                Page page = new Page(_url, _encoding);
                Visit(page);
            }
            catch
            { }

            while (true)
            {
                DBHelper oDBHelper = new DBHelper();
                string sql = "select top 1 * from tb_Visit where Visited='N' order by id DESC";
                DataTable dt = oDBHelper.Query(sql).Tables[0];

                for (int i = 0; i < dt.Rows.Count; i++)
                {
                    string id = dt.Rows[i]["ID"].ToString();
                    string url = dt.Rows[i]["Url"].ToString();

                    sql = "update tb_Visit set Visited='Y' where ID=" + id;
                    oDBHelper.Execute(sql);

                    try
                    {
                        Page oPage = new Page(url, _encoding);
                        List<string> links = oPage.GetAllLinks();
                        sql = "select url from tb_Visit where url in (";
                        for (int j = 0; j < links.Count; j++)
                        {
                            sql += "'" + links[j].Replace("'", "''") + "',";
                        }
                        if (sql.EndsWith(",")) sql = sql.Substring(0, sql.Length);
                        sql += ")";
                        DataTable dtLinks = oDBHelper.Query(sql).Tables[0];
                        for (int j = 0; j < dtLinks.Rows.Count; j++)
                        {
                            while (links.Contains(dtLinks.Rows[j]["url"].ToString()))
                                links.Remove(dtLinks.Rows[j]["url"].ToString());
                        }

                        List<string> linksDistinct = new List<string>();
                        for (int j = 0; j < links.Count; j++)
                        {
                            if (!linksDistinct.Contains(links[j]))
                            {
                                linksDistinct.Add(links[j]);
                            }
                        }

                        for (int j = 0; j < linksDistinct.Count; j++)
                        {
                            try
                            {
                                Page newPage = new Page(linksDistinct[j], _encoding);
                                Visit(newPage);
                            }
                            catch
                            { }
                        }
                    }
                    catch
                    { }
                }
            }
        }

        private Thread[] ths;

        public void StartThread(int count)
        {
            if (count <= 0)
                throw new Exception("线程数必须为正整数");

            ths = new Thread[count];
            for (int i = 0; i < ths.Length; i++)
            {
                ths[i] = new Thread(new ThreadStart(Start));
                ths[i].IsBackground = true;
                ths[i].Start();
            }
        }

        public void StopThread()
        {
            try
            {
                for (int i = 0; i < ths.Length; i++)
                {
                    ths[i].Abort();
                }
            }
            catch
            { }
        }
    }
}
